# coding=utf-8

# import requests
# import re
#
# url = 'http://www.crowdfunder.com/browse/deals'
# # html = requests.get(url).text
# # print html
#
# data = {
#     'entities_only':'true',
#     'page':'2'
# }
#
# html_post = requests.post(url,data=data)
# title = re.findall('"card-title">(.*?)</div>',html_post.text,re.S)
# for each in title:
#     print each



# 爬取计科学院课程
# 目标网站：http://www.jikexueyuan.com/course/
# 目标内容：课程名称，课程介绍，课程时间，课程等级，学习人数
# 设计知识：requests获取网页，re.sub换页，正则表达式匹配内容

import re
import requests
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

class spider(object):
    def __init__(self):
        print u'开始爬取内容。。。'

    # getsource用来获取网页源代码
    def getsource(self,url):
        html = requests.get(url)
        return html.text

    # changepage用来生产不同页数的链接
    def changepage(self,url,total_page):
        now_page = int(re.search('pageNum=(\d+)',url,re.S).group(1))
        page_group = []
        for i in range (now_page,total_page+1):
            link = re.sub('pageNum=\d+','pageNum=%s'%i,url,re.S)
            page_group.append(link)
        return page_group

    # geteveryclass用来抓取每个课程块的信息
    def geteveryclass(self,source):
        everyclass = re.findall('(<li deg="".*?</li>)',source,re.S)
        return everyclass

    # getinfo用来从每个课程块中提取出我们需要的信息
    def getinfo(self,eachclass):
        info = {}
        info['title'] = re.search('target="_blank">(>*?)</a>',eachclass,re.S).group(1)
        info['content'] = re.search('</h2><p>(.*?)</em>',eachclass,re.S).group(1)
        timeandlevel = re.findall('<em>(.*?)</em>',eachclass,re.S)
        info['classtime'] = timeandlevel[0]
        info['classlevel'] = timeandlevel[1]
        info['learning'] = re.search('"learn-number">(.*?)</em>',eachclass,re.S).group(1)
        return info

    # saveinfo用来保存结果到info.txt文件中
    def saveinfo(self,classinfo):
        f = open('info.txt','a')
        for each in classinfo:
            f.writelines('title:'+ each['title'] + '\n')
            f.writelines('content:'+ each['content'] + '\n')
            f.writelines('classtime:'+ each['ticlasstimetle'] + '\n')
            f.writelines('classlevel:'+ each['classlevel'] + '\n')
            f.writelines('learning:'+ each['learning'] + '\n\n')
        f.close()


if __name__=='__main__':

    classinfo = []
    url = 'http://www.jikexueyuan.com/course/?pageNum=1'
    jikespider = spider()
    all_links = jikespider.changepage(url,10)
    for link in all_links:
        print u'正在处理页面：' + link
        html = jikespider.getsource(link)
        everyclass = jikespider.geteveryclass(html)
        for each in everyclass:
            info = jikespider.getinfo(each)
            classinfo.append(info)

        jikespider.saveinfo(classinfo)




